# -*- coding = utf-8 -*-
# @Time : 2021/3/29 18:06
# @Author : kk_J
# @File: spider.py
# @Software: PyCharm
import json
import os
import re
import sqlite3
import time
import urllib.request
import urllib.error

import requests
from bs4 import BeautifulSoup


def get_html(link):
    head = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
                      "Chrome/88.0.4324.182 Safari/537.36 Edg/88.0.705.81 "
    }
    request = urllib.request.Request(link, headers=head)
    html = ""
    try:
        response = urllib.request.urlopen(request)
        html = response.read().decode('utf-8')
    except urllib.error.URLError as e:
        if hasattr(e, "code"):
            print(e.code)
        if hasattr(e, "reason"):
            print(e.reason)
    bs = BeautifulSoup(html, "html.parser")
    return bs


def init_db(dbpath):
    if os.path.exists(dbpath):
        print("目标数据库已存在，无需重新创建")
    else:
        sql = '''
            create table jd
            (
            id integer,
            title varchar,
            price varchar,
            specification varchar,
            views varchar,
            purchase varchar,
            comment varchar,
            frequency varchar
            )
        '''
        conn = sqlite3.connect(dbpath)
        cursor = conn.cursor()
        cursor.execute(sql)
        conn.commit()
        cursor.close()
        conn.close()


def savedata(recode, datapath, count):
    sql = '''
        insert into jd(
        id, title, price, specification, views, purchase, comment, frequency
        ) VALUES (%s)
    ''' % recode
    conn = sqlite3.connect(datapath)
    cur = conn.cursor()
    try:
        cur.execute(sql)
    except Exception as e:
        print(e)
    conn.commit()
    cur.close()
    conn.close()
    print('插入一条数据' + ' 共' + str(count) + '条')

if __name__ == '__main__':
    baseurl = 'https://wx.jdcloud.com'
    count = 1
    init_db('./data/jd.db')
    for page in range(2, 25):
        bs0 = get_html(baseurl + '/api/2_0/' + str(page))
        item_as = bs0.select('ul.list-new_v1 li.boder_v1 a')
        for item_a in item_as:
            recode = []
            item_link = baseurl + item_a.get('href')
            bs1 = get_html(item_link)

            # 数据id
            data_id = re.findall('/(\d+)$', item_a.get('href'))[0]
            data_id = '"' + data_id + '"'
            recode.append(data_id)

            # 标题
            title = bs1.select('.detail-title-new h1')[0].string
            title = '"' + title + '"'
            recode.append(title)
            # print(title)

            # 价格(元)
            price = bs1.select('.price-true')
            if len(price):
                price = price[0].string
            else:
                price = 0
            price = '"' + str(price) + '"'
            recode.append(price)
            # print(price)

            # 规格(次)
            specification = bs1.select('.price-times p')
            if specification:
                specification = specification[0].text
                specification = re.findall('^\d+', specification)
                if specification:
                    specification = specification[0]
                elif bs1.select('.price-times p'):
                    specification = bs1.select('.price-times p')[0].string
                else:
                    specification = 'null'
            else:
                specification = '0'
            specification = '"' + specification + '"'
            recode.append(specification)
            # print(specification)

            # 浏览量
            views = bs1.select('#app section.wrap div.detail-left div.order div.new-tmp .operation-list li')[0].string
            views = re.findall('\d+', views)[0]
            views = '"' + views + '"'
            recode.append(views)
            # print(views)

            # 购买量
            purchase = '0'
            try:
                url = 'https://wx.jdcloud.com/order/orderPageList?pageNow=' + str(page) + '&id=' + str(
                    re.findall('/(\d+)$', item_a.get('href'))[0]) + '&type=0'
                count_html = requests.get(url)
                purchase = re.findall('totalCount:\'(\d+)\'', count_html.text)[0]

            except Exception as e:
                print("error!!!" + title)
                print(e)
                print('购买量处错误')
            purchase = '"' + purchase + '"'
            recode.append(purchase)

            # print(purchase)

            # 正面评论量
            comment = '0'
            try:
                url = 'https://wx.jdcloud.com/comment/cpagelist?pageNow=' + str(page) + '&dataId=' + str(
                    re.findall('/(\d+)$', item_a.get('href'))[0]) + '&dataType=0&showDefault=0'
                comment_html = requests.get(url)
                comment = re.findall('totalCount:\'(\d+)\'', comment_html.text)[0]
            except Exception as e:
                print(e)
                print('正面评论量处错误')
            comment = '"' + comment + '"'
            recode.append(comment)
            # print(comment)

            # 更新频率
            frequency = re.findall('更新频率：</b><br/><br/>\s+([\u4e00-\u9fa5]+)', str(bs1.select('#intro')[0]))
            if frequency:
                frequency = frequency[0]
            else:
                frequency = '无'
            frequency = '"' + frequency + '"'
            recode.append(frequency)
            # print(frequency)

            recode = ','.join(recode)
            print('recode:' + recode)
            savedata(recode, './data/jd.db', count)
            count += 1
        time.sleep(2)
